/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * AbstractClassifierTestCase.java
 * Copyright (C) 2013 University of Waikato, Hamilton, New Zealand
 */
package moa.classifiers;

import moa.core.InstancesHeader;
import moa.core.Measurement;
import moa.evaluation.ClassificationPerformanceEvaluator;
import moa.test.AbstractTestHelper;
import moa.test.MoaTestCase;
import moa.test.TestHelper;
import moa.test.TmpFile;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.MOAUtils;
import weka.core.converters.ArffLoader;

/**
 * Ancestor for all classifier test cases.
 * 
 * @author  fracpete (fracpete at waikato dot ac dot nz)
 * @version $Revision$
 */
public abstract class AbstractClassifierTestCase
  extends MoaTestCase {

  /**
   * Container for the data collected from a classifier at a specified
   * inspection point in the stream.
   * 
   * @author  fracpete (fracpete at waikato dot ac dot nz)
   * @version $Revision$
   */
  public static class InspectionData {

    /** the inspection point. */
    public int index = -1;
    
    /** the votes. */
    public double[] votes = new double[0];
    
    /** the measurements. */
    public Measurement[] measurements = new Measurement[0];
    
    /** the model measurements. */
    public Measurement[] modelMeasurements = new Measurement[0];
    
    /**
     * Returns the container data as string.
     * 
     * @return		the string representation
     */
    @Override
    public String toString() {
      StringBuilder	result;
      int		i;
      
      result = new StringBuilder();
      
      result.append("Index\n");
      result.append("  " + index + "\n");
      
      result.append("Votes\n");
      for (i = 0; i < votes.length; i++)
	result.append("  " + i + ": " + MoaTestCase.doubleToString(votes[i], 8) + "\n");

      result.append("Measurements\n");
      for (Measurement m: measurements)
	result.append("  " + m.getName() + ": " + MoaTestCase.doubleToString(m.getValue(), 8) + "\n");

      result.append("Model measurements\n");
      for (Measurement m: modelMeasurements) {
	if (m.getName().indexOf("serialized") > -1)
	  continue;
	result.append("  " + m.getName() + ": " + MoaTestCase.doubleToString(m.getValue(), 8) + "\n");
      }
      
      return result.toString();
    }
  }
  
  /**
   * Constructs the test case. Called by subclasses.
   *
   * @param name 	the name of the test
   */
  public AbstractClassifierTestCase(String name) {
    super(name);
  }

  /**
   * Returns the test helper class to use.
   *
   * @return		the helper class instance
   */
  @Override
  protected AbstractTestHelper newTestHelper() {
    return new TestHelper(this, "moa/classifiers/data");
  }

  /**
   * Loads the data to process.
   *
   * @param filename	the filename to load (without path)
   * @param classIndex	the class index to use
   * @return		the data, null if it could not be loaded
   * @see		#getDataDirectory()
   */
  protected Instances load(String filename, int classIndex) {
    Instances	result;
    ArffLoader 	loader;
    
    result = null;
    
    try {
      loader = new ArffLoader();
      loader.setFile(new TmpFile(filename));
      result = loader.getDataSet();
      result.setClassIndex(classIndex);
    }
    catch (Exception e) {
      System.err.println("Failed to load dataset: " + filename);
      e.printStackTrace();
      result = null;
    }
    
    return result;
  }

  /**
   * Processes the input data and returns the inspection data.
   *
   * @param data		the data to work on
   * @param inspectionPoints	the inspection points
   * @param evaluator		the evaluator to use
   * @param scheme		the scheme to process the data with
   * @return			the processed data
   */
  protected InspectionData[] inspect(Instances data, int[] inspectionPoints, ClassificationPerformanceEvaluator evaluator, Classifier scheme) {
    InspectionData[]	result;
    int			i;
    int			point;
    Instance		inst;
    double[]		votes;
    
    result = new InspectionData[inspectionPoints.length];

    scheme.prepareForUse();
    
    point = 0;
    for (i = 0; i < data.numInstances(); i++) {
      inst = data.instance(i);
      
      if (i > 0) {
	votes = scheme.getVotesForInstance(inst);
	evaluator.addResult(inst, votes);

	if (point < inspectionPoints.length) {
	  if (i == inspectionPoints[point] - 1) {
	    result[point]                   = new InspectionData();
	    result[point].index             = inspectionPoints[point];
	    result[point].votes             = votes;
	    result[point].measurements      = evaluator.getPerformanceMeasurements();
	    result[point].modelMeasurements = scheme.getModelMeasurements();
	    point++;
	  }
	}
      }
      
      scheme.trainOnInstance(inst);
    }
    
    return result;
  }

  /**
   * Saves the data in the tmp directory.
   *
   * @param data	the data to save
   * @param filename	the filename to save to (without path)
   * @return		true if successfully saved
   */
  protected boolean save(Classifier cls, InspectionData[] data, String filename) {
    StringBuilder	str;
    
    str = new StringBuilder();
    
    str.append(MOAUtils.toCommandLine(cls) + "\n");
    str.append("\n");
    
    for (InspectionData d: data) {
      str.append(d.toString());
      str.append("\n");
    }
    
    return m_TestHelper.save(str, filename);
  }

  /**
   * Returns the filenames (without path) of the input data files to use
   * in the regression test.
   *
   * @return		the filenames
   */
  protected abstract String[] getRegressionInputFiles();

  /**
   * Returns the class index for the datasets.
   * 
   * @return		the class indices (0-based)
   */
  protected abstract int[] getRegressionInputClassIndex();
  
  /**
   * Returns the index of the instances in the stream to inspect the 
   * performance/classification output of the classifiers.
   * 
   * @return		the inspection indices
   */
  protected abstract int[][] getRegressionInspectionPoints();
  
  /**
   * Returns the classifier setups to use in the regression test.
   *
   * @return		the setups
   */
  protected abstract Classifier[] getRegressionClassifierSetups();

  /**
   * Returns the evaluator setups to use in the regression test.
   *
   * @return		the setups
   */
  protected abstract ClassificationPerformanceEvaluator[] getRegressionEvaluatorSetups();

  /**
   * Creates an output filename based on the input filename.
   *
   * @param input	the input filename (no path)
   * @param no		the number of the test
   * @return		the generated output filename (no path)
   */
  protected String createOutputFilename(String input, int no) {
    String	result;
    int		index;
    String	ext;

    ext = "-out" + no;

    index = input.lastIndexOf('.');
    if (index == -1) {
      result = input + ext;
    }
    else {
      result  = input.substring(0, index);
      result += ext;
      result += input.substring(index);
    }

    return result;
  }

  /**
   * Compares the processed data against previously saved output data.
   */
  public void testRegression() {
    Instances					data;
    InspectionData[]				processed;
    boolean					ok;
    String					regression;
    int						i;
    String[]					input;
    int[]					cindices;
    Classifier[]				setups;
    ClassificationPerformanceEvaluator[]	evals;
    int[][]					points;
    Classifier					current;
    String[]					output;
    TmpFile[]					outputFiles;

    if (m_NoRegressionTest)
      return;
    
    input    = getRegressionInputFiles();
    cindices = getRegressionInputClassIndex();
    output   = new String[input.length];
    setups   = getRegressionClassifierSetups();
    evals    = getRegressionEvaluatorSetups();
    points   = getRegressionInspectionPoints();
    assertEquals("Number of files and class indices differ!", input.length, cindices.length);
    assertEquals("Number of files and classifier setups differ!", input.length, setups.length);
    assertEquals("Number of classifier setups and evaluator setups differ!", setups.length, evals.length);
    assertEquals("Number of classifier setups and inspection points differ!", setups.length, points.length);

    // process data
    for (i = 0; i < input.length; i++) {
      data = load(input[i], cindices[i]);
      assertNotNull("Could not load data for regression test from " + input[i], data);

      current = setups[i].copy();
      current.prepareForUse();
      current.setModelContext(new InstancesHeader(data));
      assertNotNull("Failed to create copy of algorithm: " + MOAUtils.toCommandLine(setups[i]), current);

      processed = inspect(data, points[i], evals[i], current);
      assertNotNull("Failed to process data?", processed);

      output[i] = createOutputFilename(input[i], i);
      ok        = save(current, processed, output[i]);
      assertTrue("Failed to save regression data?", ok);
    }

    // test regression
    outputFiles = new TmpFile[output.length];
    for (i = 0; i < output.length; i++)
      outputFiles[i] = new TmpFile(output[i]);
    regression = m_Regression.compare(outputFiles);
    assertNull("Output differs:\n" + regression, regression);

    // remove output
    for (i = 0; i < output.length; i++)
      m_TestHelper.deleteFileFromTmp(output[i]);
  }
}
